#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Project: spd-sxmcc
"""
@author: lyndon
@time Created on 2019/2/27 11:44
@desc 从
"""

import os
import time
import requests
from bs4 import BeautifulSoup
from lxml import etree
from ippool.conf.settings import gen_ippool_dir, map_ip_file_prefix, ip_source_url
from ippool.lib.si_time import SiTime

t = time.time()
nowTime = lambda: str(int(round(t * 1000)))


def get_xici_ips(page=1):
    # requests的Session可以自动保持cookie,不需要自己维护cookie内容
    # page = 2
    S = requests.Session()
    target_url = ip_source_url % page
    target_headers = {'Upgrade-Insecure-Requests': '1',
                      'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
                      'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
                      'Referer': 'http://www.xicidaili.com/nn/',
                      'Accept-Encoding': 'gzip, deflate, sdch',
                      'Accept-Language': 'zh-CN,zh;q=0.8',
                      }
    target_response = S.get(url=target_url, headers=target_headers)
    target_response.encoding = 'utf-8'
    target_html = target_response.text
    bf1_ip_list = BeautifulSoup(target_html, 'lxml')
    bf2_ip_list = BeautifulSoup(str(bf1_ip_list.find_all(id='ip_list')), 'lxml')
    ip_list_info = bf2_ip_list.table.contents
    return ip_list_info


def write_file():
    map_file = os.path.join(gen_ippool_dir, map_ip_file_prefix.format(SiTime.now(), '.txt'))
    print(map_file)
    with open(map_file, 'w') as f:
        for page in range(1, 11):
            ip_list_info = get_xici_ips(page)
            for index in range(len(ip_list_info)):
                if index % 2 == 1 and index != 1:
                    dom = etree.HTML(str(ip_list_info[index]))
                    ip = dom.xpath('//td[2]')
                    port = dom.xpath('//td[3]')
                    protocol = dom.xpath('//td[6]')
                    split_flag = '\t'
                    row_flag = '\n'
                    f.write(ip[0].text + ':' + port[0].text + split_flag + protocol[0].text.lower() + split_flag + nowTime() + row_flag)
                    f.flush()
    return map_file

if __name__ == '__main__':
    write_file()

